import time
import datetime

from elasticsearch import Elasticsearch
from selenium import webdriver
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from pymongo import MongoClient
import loguru
logger = loguru.logger
es = Elasticsearch(hosts="192.168.2.117")
client = MongoClient(host='192.168.2.117', port=27017)  # 连接mongodb端口
db = client['jingdong']
mongo_collection = db['goods_info']  # crowd表格

class JD(object):
    def __init__(self,name):
        self.keys = name
        option = ChromeOptions()
        option.add_argument('--disable-infobars')  # 禁用浏览器正在被自动化程序控制的提示
        option.add_experimental_option('excludeSwitches', ['enable-automation'])
        option.add_argument('--no-sandbox')
        # option.add_argument('--headless')
        option.add_argument('--disable-dev-shm-usage')
        self.driver = webdriver.Chrome(options=option)
        self.driver.maximize_window()
        # 隐式等待
        # self.driver.implicitly_wait(10)
        url = 'http://www.jd.com'
        self.driver.get(url)
        with open('./stealth.min.js') as f:
            js = f.read()
            self.driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
                "source": js
            })
        time.sleep(3)
        kw = self.driver.find_element(By.ID,'key')
        kw.send_keys(self.keys)
        time.sleep(1)
        # 输入完后直接模拟按回车键
        kw.send_keys(Keys.RETURN)
        time.sleep(3)
        # 获取到每页的产品数量[{'第1页': 60}, {'第2页': 60}]
        self.page_count_list = []
        # 页数
        self.page = 1
        logger.info(self.driver.title)

    def product(self):
        time.sleep(1)
        # 按照销量排序
        try:
            logger.info("WebDriverWait点击销量")
            sales = WebDriverWait(self.driver, 10).until(
                EC.presence_of_element_located((By.XPATH, '//div[@id="J_filter"]/div/div[@class="f-sort"]/a[2]')))
            sales.click()
        except:
            logger.info("by_xpath点击销量")
            self.driver.find_element(By.CSS_SELECTOR,'//div[@id="J_filter"]/div/div[@class="f-sort"]/a[2]').click()
        time.sleep(4)
        # 获取一共多少页
        all_page = self.driver.find_element(By.CSS_SELECTOR,'span.p-skip>em>b').text
        while True:
            time.sleep(2)
            load_count = 1
            while True:
                try:
                    self.driver.find_element(By.ID, 'J_scroll_loading')
                    logger.info("产品正在加载...")
                    load_count += 1
                    time.sleep(2)
                    if load_count > 7:
                        logger.info("产品加载次数过多，尝试刷新页面")
                        self.driver.refresh()
                        time.sleep(3)
                        load_count = 1
                    try:
                        last_product = WebDriverWait(self.driver, 10).until(
                            EC.presence_of_element_located((By.XPATH, '//*[@id="J_goodsList"]/ul/li[last()]')))
                    except:
                        last_product = self.driver.find_element(By.XPATH, '//*[@id="J_goodsList"]/ul/li[last()]')
                    self.driver.execute_script("return arguments[0].scrollIntoView();", last_product)
                    time.sleep(3)
                except:
                    break
            try:
                logger.info("WebDriverWait准备获取下一页按钮")
                next_page = WebDriverWait(self.driver, 10).until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, 'a.pn-next')))
            except:
                # with open("第%s页报错.html" % all_page, 'w', encoding='utf-8')as f:
                #     f.write(self.driver.page_source)
                logger.error("error")
                logger.info("css_selector准备获取下一页按钮")
                next_page = self.driver.find_element(By.CSS_SELECTOR, 'a.pn-next')
            time.sleep(2)
            # 将滚动条滚动到next_page元素可见
            self.driver.execute_script("return arguments[0].scrollIntoView();", next_page)
            # self.driver.refresh()
            time.sleep(2)
            products = WebDriverWait(self.driver, 10).until(
                EC.presence_of_all_elements_located((By.CLASS_NAME, 'gl-item')))
            length = len(products)
            if length == 30 and self.page < int(all_page):
                with open(str(self.page) + '页30产品.html', 'w', encoding='utf-8')as f:
                    f.write(self.driver.page_source)
            # products = self.driver.find_elements_by_class_name('gl-item')
            # 当前第几个产品
            num = 1
            logger.info("获取所有产品%s个,共%s页" % (length, all_page))
            # 当前页获取的产品数量{'第1页': 60}
            page_count = {}
            page_count['第%s页' % self.page] = length
            for product in products[:60]:
                time.sleep(0.3)
                row = {}
                sku = product.get_attribute('data-sku')
                # 标题
                row['title'] = product.find_element(By.CSS_SELECTOR , 'div.p-name>a>em:not(.p-tag)').text
                # 价格
                row['price'] = product.find_element(By.CSS_SELECTOR, 'strong.J_%s>i' % sku).text
                # 评价数
                row['comment'] = product.find_element(By.ID, 'J_comment_%s' % sku).text + '条评价'
                try:
                    # 店铺名称
                    row['shop'] = product.find_element(By.CSS_SELECTOR,'span.J_im_icon>a').text
                    # 店铺链接
                    row['shop_url'] = product.find_element(By.CSS_SELECTOR, 'span.J_im_icon>a').get_attribute('href')
                except:
                    row['shop'] = '无店铺'
                    row['shop_url'] = '无链接'
                # 产品链接
                row['product_url'] = product.find_element(By.CSS_SELECTOR,'div.p-img>a').get_attribute('href')
                # 可领用优惠券
                all_ticket_list = product.find_elements(By.CSS_SELECTOR, 'div.p-icons>i')
                for all_ticket in all_ticket_list:
                    if all_ticket.get_attribute('data-tips') == '本商品可领用优惠券':
                        row['ticket'] = all_ticket.text
                logger.info("第%s个产品，第%s页，获取数量:%s" % (num, self.page, page_count))
                row["fetchtime"] = datetime.datetime.now()
                row["goods_genre"] = self.keys
                logger.info(row)
                mongo_collection.update_one({"_id":str(sku) +"_" + str(time.time())},{"$set":row},upsert = True)
                num += 1
                res = es.index(index="jingdong", doc_type="doc", id=sku, body=row)
            self.page_count_list.append(page_count)
            time.sleep(1)
            # 没找到pn-next disabled返回-1继续下一页，找到了pn-next disabled就终止循环
            if self.driver.page_source.find('pn-next disabled') != -1:
                break
            else:
                time.sleep(2)
                # self.driver.find_element_by_css_selector('a.pn-next').click()
                next_page.click()
                logger.info("----翻页操作----")
                logger.info(self.page_count_list)
                self.page += 1
        logger.info("处理完毕,获取情况：", self.page_count_list)

if __name__ == '__main__':
    import sys
    name = sys.argv[1] if len(sys.argv) > 1 else 'gpu'
    jd = JD(name)
    jd.product()

